Enter the directory of the maca folder on your drive and the name of the tissue you want to analyze.

tissue_of_interest = "Skin"

Load the requisite packages and some additional helper functions.

library(here)
here() starts at /Users/olgabot/code/tabula-muris
library(useful)
Loading required package: ggplot2
library(Seurat)
Loading required package: cowplot

Attaching package: 'cowplot'
The following object is masked from 'package:ggplot2':

    ggsave
Loading required package: Matrix
Warning: namespace 'Biobase' is not available and has been replaced
by .GlobalEnv when processing object 'call.'
Warning: namespace 'lme4' is not available and has been replaced
by .GlobalEnv when processing object 'call.'
Warning: namespace 'MatrixModels' is not available and has been replaced
by .GlobalEnv when processing object 'call.'
Warning: namespace 'Biobase' is not available and has been replaced
by .GlobalEnv when processing object 'call.'
Warning: namespace 'lme4' is not available and has been replaced
by .GlobalEnv when processing object 'call.'
Warning: namespace 'MatrixModels' is not available and has been replaced
by .GlobalEnv when processing object 'call.'
library(dplyr)
Warning: package 'dplyr' was built under R version 3.4.2

Attaching package: 'dplyr'
The following objects are masked from 'package:stats':

    filter, lag
The following objects are masked from 'package:base':

    intersect, setdiff, setequal, union
library(Matrix)

save_dir = here('00_data_ingest', 'tissue_robj')
# read the metadata to get the plates we want
plate_metadata_filename = here('00_data_ingest', 'facs_raw_data', 'metadata_FACS.csv')

plate_metadata <- read.csv(plate_metadata_filename, sep=",", header = TRUE)
colnames(plate_metadata)[1] <- "plate.barcode"
plate_metadata

Subset the metadata on the tissue.

tissue_plates = filter(plate_metadata, tissue == tissue_of_interest)[,c('plate.barcode','tissue','subtissue','mouse.sex')]
tissue_plates

Load the read count data.

#Load the gene names and set the metadata columns by opening the first file
filename = here('00_data_ingest', 'facs_raw_data', 'FACS', paste0(tissue_of_interest, '-counts.csv'))

raw.data = read.csv(filename, sep=",", row.names=1)
# raw.data = data.frame(row.names = rownames(raw.data))
corner(raw.data)

Make a vector of plate barcodes for each cell

plate.barcodes = lapply(colnames(raw.data), function(x) strsplit(strsplit(x, "_")[[1]][1], '.', fixed=TRUE)[[1]][2])
head(plate.barcodes)
[[1]]
[1] "MAA000427"

[[2]]
[1] "MAA000427"

[[3]]
[1] "MAA000427"

[[4]]
[1] "MAA000427"

[[5]]
[1] "MAA000427"

[[6]]
[1] "MAA000427"
MAA000427
MAA000427
MAA000427
MAA000427
MAA000427
MAA000427

Use only the metadata rows corresponding to Bladder plates. Make a plate barcode dataframe to “expand” the per-plate metadata to be per-cell.

barcode.df = t.data.frame(as.data.frame(plate.barcodes))

rownames(barcode.df) = colnames(raw.data)
colnames(barcode.df) = c('plate.barcode')
head(barcode.df)
                         plate.barcode
A12.MAA000427.3_11_M.1.1 "MAA000427"  
J12.MAA000427.3_11_M.1.1 "MAA000427"  
K14.MAA000427.3_11_M.1.1 "MAA000427"  
L16.MAA000427.3_11_M.1.1 "MAA000427"  
M18.MAA000427.3_11_M.1.1 "MAA000427"  
N20.MAA000427.3_11_M.1.1 "MAA000427"  
MAA000427

MAA000427

MAA000427

MAA000427

MAA000427

MAA000427
rnames = row.names(barcode.df)
meta.data <- merge(barcode.df, plate_metadata, by='plate.barcode', sort = F)
row.names(meta.data) <- rnames

# Sort cells by plate barcode because that's how the data was originally
meta.data = meta.data[order(meta.data$plate.barcode), ]
corner(meta.data)
raw.data = raw.data[, rownames(meta.data)]
corner(raw.data)

Process the raw data and load it into the Seurat object.

# Find ERCC's, compute the percent ERCC, and drop them from the raw data.
erccs <- grep(pattern = "^ERCC-", x = rownames(x = raw.data), value = TRUE)
percent.ercc <- Matrix::colSums(raw.data[erccs, ])/Matrix::colSums(raw.data)
ercc.index <- grep(pattern = "^ERCC-", x = rownames(x = raw.data), value = FALSE)
raw.data <- raw.data[-ercc.index,]

# Create the Seurat object with all the data
tiss <- CreateSeuratObject(raw.data = raw.data, project = tissue_of_interest, 
                    min.cells = 5, min.genes = 5)

tiss <- AddMetaData(object = tiss, meta.data)
tiss <- AddMetaData(object = tiss, percent.ercc, col.name = "percent.ercc")
# Change default name for sums of counts from nUMI to nReads
colnames(tiss@meta.data)[colnames(tiss@meta.data) == 'nUMI'] <- 'nReads'

# Create metadata columns for annotations and subannotations
tiss@meta.data[,'annotation'] <- NA
tiss@meta.data[,'subannotation'] <- NA

Calculate percent ribosomal genes.

ribo.genes <- grep(pattern = "^Rp[sl][[:digit:]]", x = rownames(x = tiss@data), value = TRUE)
percent.ribo <- Matrix::colSums(tiss@raw.data[ribo.genes, ])/Matrix::colSums(tiss@raw.data)
tiss <- AddMetaData(object = tiss, metadata = percent.ribo, col.name = "percent.ribo")

A sanity check: genes per cell vs reads per cell.

GenePlot(object = tiss, gene1 = "nReads", gene2 = "nGene", use.raw=T)

Filter out cells with few reads and few genes.

tiss <- FilterCells(object = tiss, subset.names = c("nGene", "nReads"), 
    low.thresholds = c(500, 50000), high.thresholds = c(25000, 2000000))

Normalize the data, then regress out correlation with total reads

tiss <- NormalizeData(object = tiss)
tiss <- ScaleData(object = tiss, vars.to.regress = c("nReads", "percent.ribo","Rn45s"))
[1] "Regressing out nReads"       "Regressing out percent.ribo"
[3] "Regressing out Rn45s"       

  |                                                                       
  |                                                                 |   0%
  |                                                                       
  |                                                                 |   1%
  |                                                                       
  |=                                                                |   1%
  |                                                                       
  |=                                                                |   2%
  |                                                                       
  |==                                                               |   2%
  |                                                                       
  |==                                                               |   3%
  |                                                                       
  |==                                                               |   4%
  |                                                                       
  |===                                                              |   4%
  |                                                                       
  |===                                                              |   5%
  |                                                                       
  |====                                                             |   6%
  |                                                                       
  |====                                                             |   7%
  |                                                                       
  |=====                                                            |   7%
  |                                                                       
  |=====                                                            |   8%
  |                                                                       
  |======                                                           |   9%
  |                                                                       
  |======                                                           |  10%
  |                                                                       
  |=======                                                          |  10%
  |                                                                       
  |=======                                                          |  11%
  |                                                                       
  |========                                                         |  12%
  |                                                                       
  |========                                                         |  13%
  |                                                                       
  |=========                                                        |  14%
  |                                                                       
  |==========                                                       |  15%
  |                                                                       
  |==========                                                       |  16%
  |                                                                       
  |===========                                                      |  17%
  |                                                                       
  |============                                                     |  18%
  |                                                                       
  |============                                                     |  19%
  |                                                                       
  |=============                                                    |  20%
  |                                                                       
  |==============                                                   |  21%
  |                                                                       
  |==============                                                   |  22%
  |                                                                       
  |===============                                                  |  23%
  |                                                                       
  |================                                                 |  24%
  |                                                                       
  |================                                                 |  25%
  |                                                                       
  |=================                                                |  26%
  |                                                                       
  |=================                                                |  27%
  |                                                                       
  |==================                                               |  27%
  |                                                                       
  |==================                                               |  28%
  |                                                                       
  |===================                                              |  29%
  |                                                                       
  |===================                                              |  30%
  |                                                                       
  |====================                                             |  30%
  |                                                                       
  |====================                                             |  31%
  |                                                                       
  |=====================                                            |  32%
  |                                                                       
  |=====================                                            |  33%
  |                                                                       
  |======================                                           |  33%
  |                                                                       
  |======================                                           |  34%
  |                                                                       
  |======================                                           |  35%
  |                                                                       
  |=======================                                          |  35%
  |                                                                       
  |=======================                                          |  36%
  |                                                                       
  |========================                                         |  36%
  |                                                                       
  |========================                                         |  37%
  |                                                                       
  |========================                                         |  38%
  |                                                                       
  |=========================                                        |  38%
  |                                                                       
  |=========================                                        |  39%
  |                                                                       
  |==========================                                       |  40%
  |                                                                       
  |==========================                                       |  41%
  |                                                                       
  |===========================                                      |  41%
  |                                                                       
  |===========================                                      |  42%
  |                                                                       
  |============================                                     |  43%
  |                                                                       
  |============================                                     |  44%
  |                                                                       
  |=============================                                    |  44%
  |                                                                       
  |=============================                                    |  45%
  |                                                                       
  |==============================                                   |  46%
  |                                                                       
  |==============================                                   |  47%
  |                                                                       
  |===============================                                  |  48%
  |                                                                       
  |================================                                 |  49%
  |                                                                       
  |================================                                 |  50%
  |                                                                       
  |=================================                                |  51%
  |                                                                       
  |==================================                               |  52%
  |                                                                       
  |===================================                              |  53%
  |                                                                       
  |===================================                              |  54%
  |                                                                       
  |====================================                             |  55%
  |                                                                       
  |====================================                             |  56%
  |                                                                       
  |=====================================                            |  56%
  |                                                                       
  |=====================================                            |  57%
  |                                                                       
  |======================================                           |  58%
  |                                                                       
  |======================================                           |  59%
  |                                                                       
  |=======================================                          |  59%
  |                                                                       
  |=======================================                          |  60%
  |                                                                       
  |========================================                         |  61%
  |                                                                       
  |========================================                         |  62%
  |                                                                       
  |=========================================                        |  62%
  |                                                                       
  |=========================================                        |  63%
  |                                                                       
  |=========================================                        |  64%
  |                                                                       
  |==========================================                       |  64%
  |                                                                       
  |==========================================                       |  65%
  |                                                                       
  |===========================================                      |  65%
  |                                                                       
  |===========================================                      |  66%
  |                                                                       
  |===========================================                      |  67%
  |                                                                       
  |============================================                     |  67%
  |                                                                       
  |============================================                     |  68%
  |                                                                       
  |=============================================                    |  69%
  |                                                                       
  |=============================================                    |  70%
  |                                                                       
  |==============================================                   |  70%
  |                                                                       
  |==============================================                   |  71%
  |                                                                       
  |===============================================                  |  72%
  |                                                                       
  |===============================================                  |  73%
  |                                                                       
  |================================================                 |  73%
  |                                                                       
  |================================================                 |  74%
  |                                                                       
  |=================================================                |  75%
  |                                                                       
  |=================================================                |  76%
  |                                                                       
  |==================================================               |  77%
  |                                                                       
  |===================================================              |  78%
  |                                                                       
  |===================================================              |  79%
  |                                                                       
  |====================================================             |  80%
  |                                                                       
  |=====================================================            |  81%
  |                                                                       
  |=====================================================            |  82%
  |                                                                       
  |======================================================           |  83%
  |                                                                       
  |=======================================================          |  84%
  |                                                                       
  |=======================================================          |  85%
  |                                                                       
  |========================================================         |  86%
  |                                                                       
  |=========================================================        |  87%
  |                                                                       
  |=========================================================        |  88%
  |                                                                       
  |==========================================================       |  89%
  |                                                                       
  |==========================================================       |  90%
  |                                                                       
  |===========================================================      |  90%
  |                                                                       
  |===========================================================      |  91%
  |                                                                       
  |============================================================     |  92%
  |                                                                       
  |============================================================     |  93%
  |                                                                       
  |=============================================================    |  93%
  |                                                                       
  |=============================================================    |  94%
  |                                                                       
  |==============================================================   |  95%
  |                                                                       
  |==============================================================   |  96%
  |                                                                       
  |===============================================================  |  96%
  |                                                                       
  |===============================================================  |  97%
  |                                                                       
  |===============================================================  |  98%
  |                                                                       
  |================================================================ |  98%
  |                                                                       
  |================================================================ |  99%
  |                                                                       
  |=================================================================|  99%
  |                                                                       
  |=================================================================| 100%
[1] "Scaling data matrix"

  |                                                                       
  |                                                                 |   0%
  |                                                                       
  |=================================================================| 100%
tiss <- FindVariableGenes(object = tiss, do.plot = TRUE, x.high.cutoff = Inf, y.cutoff = 0.5)

Run Principal Component Analysis.

tiss <- RunPCA(object = tiss, do.print = FALSE)
tiss <- ProjectPCA(object = tiss, do.print = FALSE)

Later on (in FindClusters and TSNE) you will pick a number of principal components to use. This has the effect of keeping the major directions of variation in the data and, ideally, supressing noise. There is no correct answer to the number to use, but a decent rule of thumb is to go until the plot plateaus.

PCElbowPlot(object = tiss)

Choose the number of principal components to use.

# Set number of principal components. 
n.pcs = 20

The clustering is performed based on a nearest neighbors graph. Cells that have similar expression will be joined together. The Louvain algorithm looks for groups of cells with high modularity–more connections within the group than between groups. The resolution parameter determines the scale…higher resolution will give more clusters, lower resolution will give fewer.

For the top-level clustering, aim to under-cluster instead of over-cluster. It will be easy to subset groups and further analyze them below.

# Set resolution 
res.used <- 0.4

tiss <- FindClusters(object = tiss, reduction.type = "pca", dims.use = 1:n.pcs, 
    resolution = res.used, print.output = 0, save.SNN = TRUE)

To visualize

# If cells are too spread out, you can raise the perplexity. If you have few cells, try a lower perplexity (but never less than 10).
tiss <- RunTSNE(object = tiss, dims.use = 1:n.pcs, seed.use = 10, perplexity=30)
# note that you can set do.label=T to help label individual clusters
TSNEPlot(object = tiss, do.label = T)

Check expression of genes of interset.

Dotplots let you see the intensity of exppression and the fraction of cells expressing for each of your genes of interest.

How big are the clusters?

table(tiss@ident)

  0   1   2   3   4   5   6 
821 493 405 262 187  50  45 

Which markers identify a specific cluster?

clust.markers <- FindMarkers(object = tiss, ident.1 = 0, only.pos = TRUE, min.pct = 0.25, thresh.use = 0.25)
print(x = head(x= clust.markers, n = 10))
               p_val  avg_diff pct.1 pct.2
Sparc   0.000000e+00 1.9711071 0.951 0.612
Sfrp1  1.238878e-282 1.8642737 0.935 0.575
Ctgf   1.199458e-253 1.6238808 0.777 0.121
Dkk3   2.448331e-235 1.3593265 0.905 0.275
Krt24  1.404468e-217 1.9063582 0.670 0.080
Alcam  1.785836e-216 0.9313602 0.864 0.232
Ftl1   1.221542e-208 0.6554612 1.000 0.949
S100a4 3.194160e-208 1.5661606 0.750 0.233
Fzd2   4.546329e-208 0.8065746 0.855 0.248
Cxcl14 1.693267e-202 1.0983006 0.993 0.881

You can also compute all markers for all clusters at once. This may take some time.

tiss.markers <- FindAllMarkers(object = tiss, only.pos = TRUE, min.pct = 0.25, thresh.use = 0.25)

Display the top markers you computed above.

tiss.markers %>% group_by(cluster) %>% top_n(5, avg_diff)

Assigning cell type identity to clusters

At a coarse level, we can use canonical markers to match the unbiased clustering to known cell types:

# stash current cluster IDs
tiss <- StashIdent(object = tiss, save.name = "cluster.ids")

# enumerate current cluster IDs and the labels for them
cluster.ids <- c(0, 1, 2, 3, 4, 5, 6)
annotation <- c("keratinocyte stem cell", "keratinocyte stem cell", "basal cell of epidermis", "epidermal cell", "epidermal cell", "keratinocyte stem cell", "stem cell of epidermis")
cell_ontology_id <- c("CL:0002337", "CL:0002337", "CL:0002187", "CL:0000362", "CL:0000362", "CL:0002337", "CL:1000428")

tiss@meta.data[,'annotation'] <- plyr::mapvalues(x = tiss@ident, from = cluster.ids, to = annotation)
tiss@meta.data[,'cell_ontology_id'] <- plyr::mapvalues(x = tiss@ident, from = cluster.ids, to = cell_ontology_id)

tiss@meta.data[tiss@cell.names,'annotation'] <- as.character(tiss@meta.data$annotation)
tiss@meta.data[tiss@cell.names,'cell_ontology_id'] <- as.character(tiss@meta.data$cell_ontology_id)

TSNEPlot(object = tiss, do.label = TRUE, pt.size = 0.5, group.by='annotation')

Checking for batch effects

Color by metadata, like plate barcode, to check for batch effects.

TSNEPlot(object = tiss, do.return = TRUE, group.by = "plate.barcode")

Print a table showing the count of cells in each identity category from each plate.

table(as.character(tiss@ident), as.character(tiss@meta.data$plate.barcode))
   
    B000126 B003283 MAA000427 MAA000435 MAA000549 MAA000597 MAA000614
  0     128     101       133       102        42        50        91
  1       3      77       101        64        31        40        58
  2       3      87        46        38        35        38        39
  3      24      49        34        31        21        13        45
  4     108      21        17         6        10         3        11
  5      47       0         1         0         1         0         0
  6       6       7        10         2         4         3         4
   
    MAA000927 MAA000938
  0        94        80
  1        52        67
  2        60        59
  3        19        26
  4         8         3
  5         1         0
  6         1         8

Save the Robject for later

When you save the annotated tissue, please give it a name.

filename = here('00_data_ingest', 'tissue_seurat_robj', 
                     paste0(tissue_of_interest, "_seurat_tiss.Robj"))
print(filename)
[1] "/Users/olgabot/code/tabula-muris/00_data_ingest/tissue_seurat_robj/Skin_seurat_tiss.Robj"
save(tiss, file=filename)
# To reload a saved object
# filename = here('00_data_ingest', 'tissue_seurat_robj', 
#                      paste0(tissue_of_interest, "_seurat_tiss.Robj"))
# load(file=filename)

Export the final metadata

So that Biohub can easily combine all your annotations, please export them as a simple csv.

head(tiss@meta.data)
filename = here('00_data_ingest', 'tissue_annotation_csv', 
                     paste0(tissue_of_interest, "_annotation.csv"))
write.csv(tiss@meta.data[,c('plate.barcode','annotation','cell_ontology_id')], file=filename)
LS0tCnRpdGxlOiAiU2tpbiBGQUNTIE5vdGVib29rIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpFbnRlciB0aGUgZGlyZWN0b3J5IG9mIHRoZSBtYWNhIGZvbGRlciBvbiB5b3VyIGRyaXZlIGFuZCB0aGUgbmFtZSBvZiB0aGUgdGlzc3VlIHlvdSB3YW50IHRvIGFuYWx5emUuCgpgYGB7cn0KdGlzc3VlX29mX2ludGVyZXN0ID0gIlNraW4iCmBgYAoKTG9hZCB0aGUgcmVxdWlzaXRlIHBhY2thZ2VzIGFuZCBzb21lIGFkZGl0aW9uYWwgaGVscGVyIGZ1bmN0aW9ucy4KCmBgYHtyfQpsaWJyYXJ5KGhlcmUpCmxpYnJhcnkodXNlZnVsKQpsaWJyYXJ5KFNldXJhdCkKbGlicmFyeShkcGx5cikKbGlicmFyeShNYXRyaXgpCgpzYXZlX2RpciA9IGhlcmUoJzAwX2RhdGFfaW5nZXN0JywgJ3Rpc3N1ZV9yb2JqJykKYGBgCgoKCmBgYHtyfQojIHJlYWQgdGhlIG1ldGFkYXRhIHRvIGdldCB0aGUgcGxhdGVzIHdlIHdhbnQKcGxhdGVfbWV0YWRhdGFfZmlsZW5hbWUgPSBoZXJlKCcwMF9kYXRhX2luZ2VzdCcsICdmYWNzX3Jhd19kYXRhJywgJ21ldGFkYXRhX0ZBQ1MuY3N2JykKCnBsYXRlX21ldGFkYXRhIDwtIHJlYWQuY3N2KHBsYXRlX21ldGFkYXRhX2ZpbGVuYW1lLCBzZXA9IiwiLCBoZWFkZXIgPSBUUlVFKQpjb2xuYW1lcyhwbGF0ZV9tZXRhZGF0YSlbMV0gPC0gInBsYXRlLmJhcmNvZGUiCnBsYXRlX21ldGFkYXRhCmBgYAoKU3Vic2V0IHRoZSBtZXRhZGF0YSBvbiB0aGUgdGlzc3VlLgoKYGBge3J9CnRpc3N1ZV9wbGF0ZXMgPSBmaWx0ZXIocGxhdGVfbWV0YWRhdGEsIHRpc3N1ZSA9PSB0aXNzdWVfb2ZfaW50ZXJlc3QpWyxjKCdwbGF0ZS5iYXJjb2RlJywndGlzc3VlJywnc3VidGlzc3VlJywnbW91c2Uuc2V4JyldCnRpc3N1ZV9wbGF0ZXMKYGBgCgpMb2FkIHRoZSByZWFkIGNvdW50IGRhdGEuCmBgYHtyfQojTG9hZCB0aGUgZ2VuZSBuYW1lcyBhbmQgc2V0IHRoZSBtZXRhZGF0YSBjb2x1bW5zIGJ5IG9wZW5pbmcgdGhlIGZpcnN0IGZpbGUKZmlsZW5hbWUgPSBoZXJlKCcwMF9kYXRhX2luZ2VzdCcsICdmYWNzX3Jhd19kYXRhJywgJ0ZBQ1MnLCBwYXN0ZTAodGlzc3VlX29mX2ludGVyZXN0LCAnLWNvdW50cy5jc3YnKSkKCnJhdy5kYXRhID0gcmVhZC5jc3YoZmlsZW5hbWUsIHNlcD0iLCIsIHJvdy5uYW1lcz0xKQojIHJhdy5kYXRhID0gZGF0YS5mcmFtZShyb3cubmFtZXMgPSByb3duYW1lcyhyYXcuZGF0YSkpCmNvcm5lcihyYXcuZGF0YSkKYGBgCk1ha2UgYSB2ZWN0b3Igb2YgcGxhdGUgYmFyY29kZXMgZm9yIGVhY2ggY2VsbAoKYGBge3J9CnBsYXRlLmJhcmNvZGVzID0gbGFwcGx5KGNvbG5hbWVzKHJhdy5kYXRhKSwgZnVuY3Rpb24oeCkgc3Ryc3BsaXQoc3Ryc3BsaXQoeCwgIl8iKVtbMV1dWzFdLCAnLicsIGZpeGVkPVRSVUUpW1sxXV1bMl0pCmhlYWQocGxhdGUuYmFyY29kZXMpCmBgYAoKVXNlIG9ubHkgdGhlIG1ldGFkYXRhIHJvd3MgY29ycmVzcG9uZGluZyB0byBCbGFkZGVyIHBsYXRlcy4gTWFrZSBhIHBsYXRlIGJhcmNvZGUgZGF0YWZyYW1lIHRvICJleHBhbmQiIHRoZSBwZXItcGxhdGUgbWV0YWRhdGEgdG8gYmUgcGVyLWNlbGwuCmBgYHtyfQpiYXJjb2RlLmRmID0gdC5kYXRhLmZyYW1lKGFzLmRhdGEuZnJhbWUocGxhdGUuYmFyY29kZXMpKQoKcm93bmFtZXMoYmFyY29kZS5kZikgPSBjb2xuYW1lcyhyYXcuZGF0YSkKY29sbmFtZXMoYmFyY29kZS5kZikgPSBjKCdwbGF0ZS5iYXJjb2RlJykKaGVhZChiYXJjb2RlLmRmKQoKcm5hbWVzID0gcm93Lm5hbWVzKGJhcmNvZGUuZGYpCm1ldGEuZGF0YSA8LSBtZXJnZShiYXJjb2RlLmRmLCBwbGF0ZV9tZXRhZGF0YSwgYnk9J3BsYXRlLmJhcmNvZGUnLCBzb3J0ID0gRikKcm93Lm5hbWVzKG1ldGEuZGF0YSkgPC0gcm5hbWVzCgojIFNvcnQgY2VsbHMgYnkgcGxhdGUgYmFyY29kZSBiZWNhdXNlIHRoYXQncyBob3cgdGhlIGRhdGEgd2FzIG9yaWdpbmFsbHkKbWV0YS5kYXRhID0gbWV0YS5kYXRhW29yZGVyKG1ldGEuZGF0YSRwbGF0ZS5iYXJjb2RlKSwgXQpjb3JuZXIobWV0YS5kYXRhKQpyYXcuZGF0YSA9IHJhdy5kYXRhWywgcm93bmFtZXMobWV0YS5kYXRhKV0KY29ybmVyKHJhdy5kYXRhKQpgYGAKClByb2Nlc3MgdGhlIHJhdyBkYXRhIGFuZCBsb2FkIGl0IGludG8gdGhlIFNldXJhdCBvYmplY3QuCgpgYGB7cn0KIyBGaW5kIEVSQ0MncywgY29tcHV0ZSB0aGUgcGVyY2VudCBFUkNDLCBhbmQgZHJvcCB0aGVtIGZyb20gdGhlIHJhdyBkYXRhLgplcmNjcyA8LSBncmVwKHBhdHRlcm4gPSAiXkVSQ0MtIiwgeCA9IHJvd25hbWVzKHggPSByYXcuZGF0YSksIHZhbHVlID0gVFJVRSkKcGVyY2VudC5lcmNjIDwtIE1hdHJpeDo6Y29sU3VtcyhyYXcuZGF0YVtlcmNjcywgXSkvTWF0cml4Ojpjb2xTdW1zKHJhdy5kYXRhKQplcmNjLmluZGV4IDwtIGdyZXAocGF0dGVybiA9ICJeRVJDQy0iLCB4ID0gcm93bmFtZXMoeCA9IHJhdy5kYXRhKSwgdmFsdWUgPSBGQUxTRSkKcmF3LmRhdGEgPC0gcmF3LmRhdGFbLWVyY2MuaW5kZXgsXQoKIyBDcmVhdGUgdGhlIFNldXJhdCBvYmplY3Qgd2l0aCBhbGwgdGhlIGRhdGEKdGlzcyA8LSBDcmVhdGVTZXVyYXRPYmplY3QocmF3LmRhdGEgPSByYXcuZGF0YSwgcHJvamVjdCA9IHRpc3N1ZV9vZl9pbnRlcmVzdCwgCiAgICAgICAgICAgICAgICAgICAgbWluLmNlbGxzID0gNSwgbWluLmdlbmVzID0gNSkKCnRpc3MgPC0gQWRkTWV0YURhdGEob2JqZWN0ID0gdGlzcywgbWV0YS5kYXRhKQp0aXNzIDwtIEFkZE1ldGFEYXRhKG9iamVjdCA9IHRpc3MsIHBlcmNlbnQuZXJjYywgY29sLm5hbWUgPSAicGVyY2VudC5lcmNjIikKIyBDaGFuZ2UgZGVmYXVsdCBuYW1lIGZvciBzdW1zIG9mIGNvdW50cyBmcm9tIG5VTUkgdG8gblJlYWRzCmNvbG5hbWVzKHRpc3NAbWV0YS5kYXRhKVtjb2xuYW1lcyh0aXNzQG1ldGEuZGF0YSkgPT0gJ25VTUknXSA8LSAnblJlYWRzJwoKIyBDcmVhdGUgbWV0YWRhdGEgY29sdW1ucyBmb3IgYW5ub3RhdGlvbnMgYW5kIHN1YmFubm90YXRpb25zCnRpc3NAbWV0YS5kYXRhWywnYW5ub3RhdGlvbiddIDwtIE5BCnRpc3NAbWV0YS5kYXRhWywnc3ViYW5ub3RhdGlvbiddIDwtIE5BCmBgYAoKCkNhbGN1bGF0ZSBwZXJjZW50IHJpYm9zb21hbCBnZW5lcy4KCmBgYHtyfQpyaWJvLmdlbmVzIDwtIGdyZXAocGF0dGVybiA9ICJeUnBbc2xdW1s6ZGlnaXQ6XV0iLCB4ID0gcm93bmFtZXMoeCA9IHRpc3NAZGF0YSksIHZhbHVlID0gVFJVRSkKcGVyY2VudC5yaWJvIDwtIE1hdHJpeDo6Y29sU3Vtcyh0aXNzQHJhdy5kYXRhW3JpYm8uZ2VuZXMsIF0pL01hdHJpeDo6Y29sU3Vtcyh0aXNzQHJhdy5kYXRhKQp0aXNzIDwtIEFkZE1ldGFEYXRhKG9iamVjdCA9IHRpc3MsIG1ldGFkYXRhID0gcGVyY2VudC5yaWJvLCBjb2wubmFtZSA9ICJwZXJjZW50LnJpYm8iKQpgYGAKCkEgc2FuaXR5IGNoZWNrOiBnZW5lcyBwZXIgY2VsbCB2cyByZWFkcyBwZXIgY2VsbC4KCmBgYHtyfQpHZW5lUGxvdChvYmplY3QgPSB0aXNzLCBnZW5lMSA9ICJuUmVhZHMiLCBnZW5lMiA9ICJuR2VuZSIsIHVzZS5yYXc9VCkKYGBgCgpGaWx0ZXIgb3V0IGNlbGxzIHdpdGggZmV3IHJlYWRzIGFuZCBmZXcgZ2VuZXMuCgpgYGB7cn0KdGlzcyA8LSBGaWx0ZXJDZWxscyhvYmplY3QgPSB0aXNzLCBzdWJzZXQubmFtZXMgPSBjKCJuR2VuZSIsICJuUmVhZHMiKSwgCiAgICBsb3cudGhyZXNob2xkcyA9IGMoNTAwLCA1MDAwMCksIGhpZ2gudGhyZXNob2xkcyA9IGMoMjUwMDAsIDIwMDAwMDApKQpgYGAKCgpOb3JtYWxpemUgdGhlIGRhdGEsIHRoZW4gcmVncmVzcyBvdXQgY29ycmVsYXRpb24gd2l0aCB0b3RhbCByZWFkcwpgYGB7cn0KdGlzcyA8LSBOb3JtYWxpemVEYXRhKG9iamVjdCA9IHRpc3MpCnRpc3MgPC0gU2NhbGVEYXRhKG9iamVjdCA9IHRpc3MsIHZhcnMudG8ucmVncmVzcyA9IGMoIm5SZWFkcyIsICJwZXJjZW50LnJpYm8iLCJSbjQ1cyIpKQp0aXNzIDwtIEZpbmRWYXJpYWJsZUdlbmVzKG9iamVjdCA9IHRpc3MsIGRvLnBsb3QgPSBUUlVFLCB4LmhpZ2guY3V0b2ZmID0gSW5mLCB5LmN1dG9mZiA9IDAuNSkKYGBgCgoKUnVuIFByaW5jaXBhbCBDb21wb25lbnQgQW5hbHlzaXMuCmBgYHtyfQp0aXNzIDwtIFJ1blBDQShvYmplY3QgPSB0aXNzLCBkby5wcmludCA9IEZBTFNFKQp0aXNzIDwtIFByb2plY3RQQ0Eob2JqZWN0ID0gdGlzcywgZG8ucHJpbnQgPSBGQUxTRSkKYGBgCgpgYGB7ciwgZWNobz1GQUxTRSwgZmlnLmhlaWdodD00LCBmaWcud2lkdGg9OH0KUENIZWF0bWFwKG9iamVjdCA9IHRpc3MsIHBjLnVzZSA9IDE6MywgY2VsbHMudXNlID0gNTAwLCBkby5iYWxhbmNlZCA9IFRSVUUsIGxhYmVsLmNvbHVtbnMgPSBGQUxTRSwgbnVtLmdlbmVzID0gOCkKYGBgCgpMYXRlciBvbiAoaW4gRmluZENsdXN0ZXJzIGFuZCBUU05FKSB5b3Ugd2lsbCBwaWNrIGEgbnVtYmVyIG9mIHByaW5jaXBhbCBjb21wb25lbnRzIHRvIHVzZS4gVGhpcyBoYXMgdGhlIGVmZmVjdCBvZiBrZWVwaW5nIHRoZSBtYWpvciBkaXJlY3Rpb25zIG9mIHZhcmlhdGlvbiBpbiB0aGUgZGF0YSBhbmQsIGlkZWFsbHksIHN1cHJlc3Npbmcgbm9pc2UuIFRoZXJlIGlzIG5vIGNvcnJlY3QgYW5zd2VyIHRvIHRoZSBudW1iZXIgdG8gdXNlLCBidXQgYSBkZWNlbnQgcnVsZSBvZiB0aHVtYiBpcyB0byBnbyB1bnRpbCB0aGUgcGxvdCBwbGF0ZWF1cy4KCmBgYHtyfQpQQ0VsYm93UGxvdChvYmplY3QgPSB0aXNzKQpgYGAKCkNob29zZSB0aGUgbnVtYmVyIG9mIHByaW5jaXBhbCBjb21wb25lbnRzIHRvIHVzZS4KYGBge3J9CiMgU2V0IG51bWJlciBvZiBwcmluY2lwYWwgY29tcG9uZW50cy4gCm4ucGNzID0gMjAKYGBgCgoKVGhlIGNsdXN0ZXJpbmcgaXMgcGVyZm9ybWVkIGJhc2VkIG9uIGEgbmVhcmVzdCBuZWlnaGJvcnMgZ3JhcGguIENlbGxzIHRoYXQgaGF2ZSBzaW1pbGFyIGV4cHJlc3Npb24gd2lsbCBiZSBqb2luZWQgdG9nZXRoZXIuIFRoZSBMb3V2YWluIGFsZ29yaXRobSBsb29rcyBmb3IgZ3JvdXBzIG9mIGNlbGxzIHdpdGggaGlnaCBtb2R1bGFyaXR5LS1tb3JlIGNvbm5lY3Rpb25zIHdpdGhpbiB0aGUgZ3JvdXAgdGhhbiBiZXR3ZWVuIGdyb3Vwcy4gVGhlIHJlc29sdXRpb24gcGFyYW1ldGVyIGRldGVybWluZXMgdGhlIHNjYWxlLi4uaGlnaGVyIHJlc29sdXRpb24gd2lsbCBnaXZlIG1vcmUgY2x1c3RlcnMsIGxvd2VyIHJlc29sdXRpb24gd2lsbCBnaXZlIGZld2VyLgoKRm9yIHRoZSB0b3AtbGV2ZWwgY2x1c3RlcmluZywgYWltIHRvIHVuZGVyLWNsdXN0ZXIgaW5zdGVhZCBvZiBvdmVyLWNsdXN0ZXIuIEl0IHdpbGwgYmUgZWFzeSB0byBzdWJzZXQgZ3JvdXBzIGFuZCBmdXJ0aGVyIGFuYWx5emUgdGhlbSBiZWxvdy4KCmBgYHtyfQojIFNldCByZXNvbHV0aW9uIApyZXMudXNlZCA8LSAwLjQKCnRpc3MgPC0gRmluZENsdXN0ZXJzKG9iamVjdCA9IHRpc3MsIHJlZHVjdGlvbi50eXBlID0gInBjYSIsIGRpbXMudXNlID0gMTpuLnBjcywgCiAgICByZXNvbHV0aW9uID0gcmVzLnVzZWQsIHByaW50Lm91dHB1dCA9IDAsIHNhdmUuU05OID0gVFJVRSkKYGBgCgpUbyB2aXN1YWxpemUgCmBgYHtyfQojIElmIGNlbGxzIGFyZSB0b28gc3ByZWFkIG91dCwgeW91IGNhbiByYWlzZSB0aGUgcGVycGxleGl0eS4gSWYgeW91IGhhdmUgZmV3IGNlbGxzLCB0cnkgYSBsb3dlciBwZXJwbGV4aXR5IChidXQgbmV2ZXIgbGVzcyB0aGFuIDEwKS4KdGlzcyA8LSBSdW5UU05FKG9iamVjdCA9IHRpc3MsIGRpbXMudXNlID0gMTpuLnBjcywgc2VlZC51c2UgPSAxMCwgcGVycGxleGl0eT0zMCkKYGBgCgpgYGB7cn0KIyBub3RlIHRoYXQgeW91IGNhbiBzZXQgZG8ubGFiZWw9VCB0byBoZWxwIGxhYmVsIGluZGl2aWR1YWwgY2x1c3RlcnMKVFNORVBsb3Qob2JqZWN0ID0gdGlzcywgZG8ubGFiZWwgPSBUKQpgYGAKCkNoZWNrIGV4cHJlc3Npb24gb2YgZ2VuZXMgb2YgaW50ZXJzZXQuCgpgYGB7ciwgZWNobz1GQUxTRSwgZmlnLmhlaWdodD0yOCwgZmlnLndpZHRoPTEwfQpnZW5lc190b19jaGVjayA9IGMoJ0NkMzQnLCdJdGdhNicsJ0tydDE0JywnS3J0MTUnLCdMZ3I1JywnTGdyNicsJ1NveDknLCdMcmlnMScsJ0dsaTEnLCdBdHhuMScsJ1Nic24nLCdEa2szJywnV2lmMScsJ0ZnZjE4JywnU3BhcmMnLCdBc3BuJywnUG9zdG4nLCdTY2QxJywnTWdzdDEnLCdLcnQxMCcsJ0xvcicsJ010NCcsJ0NkM2cnLCdDdHNzJywnQ2QyMDcnLCdLcnQ2YScsJ0tydDZiJywnQ2RoMycpCiNQbGV0MSBtaXNzaW5nCkZlYXR1cmVQbG90KHRpc3MsIGdlbmVzX3RvX2NoZWNrLCBwdC5zaXplID0gMC41LCBuQ29sID0gMykKYGBgCgpEb3RwbG90cyBsZXQgeW91IHNlZSB0aGUgaW50ZW5zaXR5IG9mIGV4cHByZXNzaW9uIGFuZCB0aGUgZnJhY3Rpb24gb2YgY2VsbHMgZXhwcmVzc2luZyBmb3IgZWFjaCBvZiB5b3VyIGdlbmVzIG9mIGludGVyZXN0LgoKYGBge3IsIGVjaG89RkFMU0UsIGZpZy5oZWlnaHQ9NCwgZmlnLndpZHRoPTEyfQojIFRvIGNoYW5nZSB0aGUgeS1heGlzIHRvIHNob3cgcmF3IGNvdW50cywgYWRkIHVzZS5yYXcgPSBULgpEb3RQbG90KHRpc3MsIGdlbmVzX3RvX2NoZWNrLCBwbG90LmxlZ2VuZCA9IFQpCmBgYAoKYGBge3IsIGVjaG89RkFMU0UsIGZpZy5oZWlnaHQ9MTYsIGZpZy53aWR0aD0xMn0KVmxuUGxvdCh0aXNzLCBnZW5lc190b19jaGVjaywgcG9pbnQuc2l6ZS51c2UgPSAwLjUpCmBgYAoKSG93IGJpZyBhcmUgdGhlIGNsdXN0ZXJzPwpgYGB7cn0KdGFibGUodGlzc0BpZGVudCkKYGBgCgpXaGljaCBtYXJrZXJzIGlkZW50aWZ5IGEgc3BlY2lmaWMgY2x1c3Rlcj8KCmBgYHtyfQpjbHVzdC5tYXJrZXJzIDwtIEZpbmRNYXJrZXJzKG9iamVjdCA9IHRpc3MsIGlkZW50LjEgPSAwLCBvbmx5LnBvcyA9IFRSVUUsIG1pbi5wY3QgPSAwLjI1LCB0aHJlc2gudXNlID0gMC4yNSkKYGBgCgpgYGB7cn0KcHJpbnQoeCA9IGhlYWQoeD0gY2x1c3QubWFya2VycywgbiA9IDEwKSkKYGBgCgpZb3UgY2FuIGFsc28gY29tcHV0ZSBhbGwgbWFya2VycyBmb3IgYWxsIGNsdXN0ZXJzIGF0IG9uY2UuIFRoaXMgbWF5IHRha2Ugc29tZSB0aW1lLgpgYGB7cn0KdGlzcy5tYXJrZXJzIDwtIEZpbmRBbGxNYXJrZXJzKG9iamVjdCA9IHRpc3MsIG9ubHkucG9zID0gVFJVRSwgbWluLnBjdCA9IDAuMjUsIHRocmVzaC51c2UgPSAwLjI1KQpgYGAKCkRpc3BsYXkgdGhlIHRvcCBtYXJrZXJzIHlvdSBjb21wdXRlZCBhYm92ZS4KYGBge3J9CnRpc3MubWFya2VycyAlPiUgZ3JvdXBfYnkoY2x1c3RlcikgJT4lIHRvcF9uKDUsIGF2Z19kaWZmKQpgYGAKCiMjIEFzc2lnbmluZyBjZWxsIHR5cGUgaWRlbnRpdHkgdG8gY2x1c3RlcnMKCkF0IGEgY29hcnNlIGxldmVsLCB3ZSBjYW4gdXNlIGNhbm9uaWNhbCBtYXJrZXJzIHRvIG1hdGNoIHRoZSB1bmJpYXNlZCBjbHVzdGVyaW5nIHRvIGtub3duIGNlbGwgdHlwZXM6CgpgYGB7cn0KIyBzdGFzaCBjdXJyZW50IGNsdXN0ZXIgSURzCnRpc3MgPC0gU3Rhc2hJZGVudChvYmplY3QgPSB0aXNzLCBzYXZlLm5hbWUgPSAiY2x1c3Rlci5pZHMiKQoKIyBlbnVtZXJhdGUgY3VycmVudCBjbHVzdGVyIElEcyBhbmQgdGhlIGxhYmVscyBmb3IgdGhlbQpjbHVzdGVyLmlkcyA8LSBjKDAsIDEsIDIsIDMsIDQsIDUsIDYpCmFubm90YXRpb24gPC0gYygia2VyYXRpbm9jeXRlIHN0ZW0gY2VsbCIsICJrZXJhdGlub2N5dGUgc3RlbSBjZWxsIiwgImJhc2FsIGNlbGwgb2YgZXBpZGVybWlzIiwgImVwaWRlcm1hbCBjZWxsIiwgImVwaWRlcm1hbCBjZWxsIiwgImtlcmF0aW5vY3l0ZSBzdGVtIGNlbGwiLCAic3RlbSBjZWxsIG9mIGVwaWRlcm1pcyIpCmNlbGxfb250b2xvZ3lfaWQgPC0gYygiQ0w6MDAwMjMzNyIsICJDTDowMDAyMzM3IiwgIkNMOjAwMDIxODciLCAiQ0w6MDAwMDM2MiIsICJDTDowMDAwMzYyIiwgIkNMOjAwMDIzMzciLCAiQ0w6MTAwMDQyOCIpCgp0aXNzQG1ldGEuZGF0YVssJ2Fubm90YXRpb24nXSA8LSBwbHlyOjptYXB2YWx1ZXMoeCA9IHRpc3NAaWRlbnQsIGZyb20gPSBjbHVzdGVyLmlkcywgdG8gPSBhbm5vdGF0aW9uKQp0aXNzQG1ldGEuZGF0YVssJ2NlbGxfb250b2xvZ3lfaWQnXSA8LSBwbHlyOjptYXB2YWx1ZXMoeCA9IHRpc3NAaWRlbnQsIGZyb20gPSBjbHVzdGVyLmlkcywgdG8gPSBjZWxsX29udG9sb2d5X2lkKQoKdGlzc0BtZXRhLmRhdGFbdGlzc0BjZWxsLm5hbWVzLCdhbm5vdGF0aW9uJ10gPC0gYXMuY2hhcmFjdGVyKHRpc3NAbWV0YS5kYXRhJGFubm90YXRpb24pCnRpc3NAbWV0YS5kYXRhW3Rpc3NAY2VsbC5uYW1lcywnY2VsbF9vbnRvbG9neV9pZCddIDwtIGFzLmNoYXJhY3Rlcih0aXNzQG1ldGEuZGF0YSRjZWxsX29udG9sb2d5X2lkKQoKVFNORVBsb3Qob2JqZWN0ID0gdGlzcywgZG8ubGFiZWwgPSBUUlVFLCBwdC5zaXplID0gMC41LCBncm91cC5ieT0nYW5ub3RhdGlvbicpCmBgYAoKCiMjIENoZWNraW5nIGZvciBiYXRjaCBlZmZlY3RzCgoKQ29sb3IgYnkgbWV0YWRhdGEsIGxpa2UgcGxhdGUgYmFyY29kZSwgdG8gY2hlY2sgZm9yIGJhdGNoIGVmZmVjdHMuCmBgYHtyfQpUU05FUGxvdChvYmplY3QgPSB0aXNzLCBkby5yZXR1cm4gPSBUUlVFLCBncm91cC5ieSA9ICJwbGF0ZS5iYXJjb2RlIikKYGBgCgpQcmludCBhIHRhYmxlIHNob3dpbmcgdGhlIGNvdW50IG9mIGNlbGxzIGluIGVhY2ggaWRlbnRpdHkgY2F0ZWdvcnkgZnJvbSBlYWNoIHBsYXRlLgoKYGBge3J9CnRhYmxlKGFzLmNoYXJhY3Rlcih0aXNzQGlkZW50KSwgYXMuY2hhcmFjdGVyKHRpc3NAbWV0YS5kYXRhJHBsYXRlLmJhcmNvZGUpKQpgYGAKCiMgU2F2ZSB0aGUgUm9iamVjdCBmb3IgbGF0ZXIKV2hlbiB5b3Ugc2F2ZSB0aGUgYW5ub3RhdGVkIHRpc3N1ZSwgcGxlYXNlIGdpdmUgaXQgYSBuYW1lLgoKYGBge3J9CmZpbGVuYW1lID0gaGVyZSgnMDBfZGF0YV9pbmdlc3QnLCAndGlzc3VlX3NldXJhdF9yb2JqJywgCiAgICAgICAgICAgICAgICAgICAgIHBhc3RlMCh0aXNzdWVfb2ZfaW50ZXJlc3QsICJfc2V1cmF0X3Rpc3MuUm9iaiIpKQpwcmludChmaWxlbmFtZSkKc2F2ZSh0aXNzLCBmaWxlPWZpbGVuYW1lKQpgYGAKCmBgYHtyfQojIFRvIHJlbG9hZCBhIHNhdmVkIG9iamVjdAojIGZpbGVuYW1lID0gaGVyZSgnMDBfZGF0YV9pbmdlc3QnLCAndGlzc3VlX3NldXJhdF9yb2JqJywgCiMgICAgICAgICAgICAgICAgICAgICAgcGFzdGUwKHRpc3N1ZV9vZl9pbnRlcmVzdCwgIl9zZXVyYXRfdGlzcy5Sb2JqIikpCiMgbG9hZChmaWxlPWZpbGVuYW1lKQpgYGAKCgoKIyBFeHBvcnQgdGhlIGZpbmFsIG1ldGFkYXRhCgpTbyB0aGF0IEJpb2h1YiBjYW4gZWFzaWx5IGNvbWJpbmUgYWxsIHlvdXIgYW5ub3RhdGlvbnMsIHBsZWFzZSBleHBvcnQgdGhlbSBhcyBhIHNpbXBsZSBjc3YuCgpgYGB7cn0KaGVhZCh0aXNzQG1ldGEuZGF0YSkKYGBgCgoKYGBge3J9CmZpbGVuYW1lID0gaGVyZSgnMDBfZGF0YV9pbmdlc3QnLCAndGlzc3VlX2Fubm90YXRpb25fY3N2JywgCiAgICAgICAgICAgICAgICAgICAgIHBhc3RlMCh0aXNzdWVfb2ZfaW50ZXJlc3QsICJfYW5ub3RhdGlvbi5jc3YiKSkKd3JpdGUuY3N2KHRpc3NAbWV0YS5kYXRhWyxjKCdwbGF0ZS5iYXJjb2RlJywnYW5ub3RhdGlvbicsJ2NlbGxfb250b2xvZ3lfaWQnKV0sIGZpbGU9ZmlsZW5hbWUpCmBgYAoKCgo=